############################################################################
#
#  Program:  OpenSees
#
#  Purpose:  A Top-level Makefile to create the libraries needed
#	     to use the OpenSees framework. 
#            and below.
#
#  Written: fmk 
#  Created: 10/99
#
#  Send bug reports, comments or suggestions to fmckenna@ce.berkeley.edu
#
############################################################################

# Works on intel machine running Mac OS X.4.10 with XCode 2.4.1 with 

# need to install gFortran from
# http://www.macresearch.org/xcode_gfortran_plugin_update
# Now USING hpc.sourceforge.net

# FOR PARALLEL
#1.  install openmpi (i used  openmpi-v1.2.6.tar.gz, configured with:
#     configure --prefix=/usr/local/openmpi-1.2.6
#2.  use scalapack installer to install BLAS, BLACS, lapack & scalapack
#3.   http://netlib.org/scalapack/scalapack_installer.tgz
#     tar xvfz scalapack_installer.tgz
#     cd scalapack_installer_0.91
# ./setup.py --downblas --downblacs --downlapack --mpibindir=/usr/local/openmpi-1.2.6/bin --mpicc=/usr/local/openmpi-1.2.6/bin/mpicc --mp#if77=/usr/local/openmpi-1.2.6/bin/mpif77 --mpiincdir=/usr/local/openmpi-1.2.6/include
#     mv scalapack_0.9.1 /usr/local (if not already there)
#4. get MUMPS
#   create a Makefile.inc (either take and modify one in Make.inc) or use mine (and modify):
#   OpenSees/SRC/systemof_eqn/linearSOE/mumps/Makefile.inc.MacOSx
 
#?. install blas (i used goto blas from tacc v1.17)
#      make F_COMPILER=gfortran BINARY32=1 SMP=1
#      ATLAS ~/Desktop/ATLAS/configure -b 32 -t 0
#      make; make install (installed in /usr/local/atlas)
#     
# And of Course Tcl from ActiveState - i used 8.5.5 version


# %---------------------------------%
# |  SECTION 1: PROGRAM             |
# %---------------------------------%
#
# Specify the location and name of the OpenSees interpreter program
# that will be created (if this all works!)

PROGRAMMING_MODE = SEQUENTIAL
#PROGRAMMING_MODE = PARALLEL
#PROGRAMMING_MODE = PARALLEL_INTERPRETERS

OpenSees_PROGRAM = $(HOME)/bin/OpenSees

ifeq ($(PROGRAMMING_MODE), PARALLEL)
OpenSees_PROGRAM = $(HOME)/bin/OpenSeesSP
endif

ifeq ($(PROGRAMMING_MODE), PARALLEL_INTERPRETERS)
OpenSees_PROGRAM = $(HOME)/bin/OpenSeesMP
endif

OPERATING_SYSTEM = LINUX

DEBUG_MODE = NO_DEBUG
#DEBUG_MODE = DEBUG
#RELIABILITY = YES_RELIABILITY
RELIABILITY = NO_RELIABILITY

BASE		= /usr/local
HOME		= /Users/fmk
FE		= $(HOME)/OpenSees/SRC

GRAPHICS = UsingOpenGL
GRAPHIC_FLAG = -D_AGL
GRAPHIC_LIBRARY     = -framework AGL -framework OpenGL -framework Carbon
AGL_OBJS = $(FE)/renderer/AGL_Device.o


#HAVE_BLAS = YES

ifeq ($(PROGRAMMING_MODE), SEQUENTIAL)
HAVE_BLAS = NO
endif

BLASdir      = $(HOME)/OpenSees/OTHER/BLAS
CBLASdir     = $(HOME)/OpenSees/OTHER/CBLAS

BLAS_LIBRARY    = $(HOME)/lib/libBlas.a
CBLAS_LIBRARY   = $(HOME)/lib/libCBlas.a


ifeq ($(HAVE_BLAS), YES) 

BLASdir =
BLAS_LIBRARY = 
CBLAS_LIBRARY = 

endif

AMDdir    = $(HOME)/OpenSees/OTHER/AMD
LAPACKdir    = $(HOME)/OpenSees/OTHER/LAPACK
SUPERLUdir   = $(HOME)/OpenSees/OTHER/SuperLU_4.1/SRC
SUPERLU_DISTdir   = $(HOME)/OpenSees/OTHER/SuperLU_DIST_2.5/SRC
ARPACKdir    = $(HOME)/OpenSees/OTHER/ARPACK
UMFPACKdir   = $(HOME)/OpenSees/OTHER/UMFPACK
METISdir       = $(HOME)/OpenSees/OTHER/METIS
ITPACKdir       = $(HOME)/OpenSees/OTHER/ITPACK
CSPARSEdir = $(HOME)/OpenSees/OTHER/CSPARSE

SRCdir       = $(HOME)/OpenSees/SRC

DIRS        = $(BLASdir) \
	$(CBLASdir) \
	$(LAPACKdir) \
	$(AMDdir) \
	$(SUPERLUdir) \
	$(SUPERLU_DISTdir) \
	$(ARPACKdir) \
	$(UMFPACKdir) \
	$(METISdir) \
	$(CSPARSEdir) \
	$(ITPACKdir) \
	$(SRCdir)


DISTRIBUTED_SUPERLU_LIBRARY     = $(HOME)/lib/libDistributedSuperLU.a

ifeq ($(PROGRAMMING_MODE), SEQUENTIAL)
DIRS        = $(BLASdir) \
	$(CBLASdir) \
	$(LAPACKdir) \
	$(AMDdir) \
	$(SUPERLUdir) \
	$(ARPACKdir) \
	$(UMFPACKdir) \
	$(CSPARSEdir) \
	$(METISdir) \
	$(ITPACKdir) \
	$(SRCdir)

DISTRIBUTED_SUPERLU_LIBRARY     = 

endif

ifeq ($(PROGRAMMING_MODE), PARALLEL)

GRAPHICS = NONE
GRAPHIC_FLAG = -D_NOGRAPHICS
GRAPHIC_LIBRARY     = 
AGL_OBJS = 

PROGRAMMING_FLAG = -D_PARALLEL_PROCESSING

DIRS        = $(SUPERLUdir) \
	      $(SUPERLU_DISTdir) \
	      $(ARPACKdir) \
	      $(AMDdir) \
              $(LAPACKdir) \
              $(UMFPACKdir) \
              $(METISdir) \
	      $(BLASdir) $(CBLASdir)  \
	      $(CSPARSEdir) \
              $(SRCdir) 

endif

ifeq ($(PROGRAMMING_MODE), PARALLEL_INTERPRETERS)

PROGRAMMING_FLAG = -D_PARALLEL_INTERPRETERS

GRAPHICS = NONE
GRAPHIC_FLAG = -D_NOGRAPHICS
GRAPHIC_LIBRARY     = 
AGL_OBJS = 

DIRS        = $(SUPERLUdir) \
	      $(SUPERLU_DISTdir) \
              $(LAPACKdir) \
              $(ARPACKdir) \
	      $(AMDdir) \
              $(UMFPACKdir) \
              $(METISdir) \
	      $(ITPACKdir) \
	      $(BLASdir) $(CBLASdir)  \
	      $(CSPARSEdir) \
              $(SRCdir) 
endif


# %-------------------------------------------------------%
# | SECTION 3: LIBRARIES                                  |
# |                                                       |
# | The following section defines the libraries that will |
# | be created and/or linked with when the libraries are  | 
# | being created or linked with.                         |
# %-------------------------------------------------------%
#
# Note: if vendor supplied BLAS and LAPACK libraries leave the
# libraries blank. You have to get your own copy of the tcl/tk 
# library!! 
#
# Note: For libraries that will be created (any in DIRS above)
# make sure the directory exsists where you want the library to go!

FE_LIBRARY      = $(HOME)/lib/libOpenSees.a
NDARRAY_LIBRARY = $(HOME)/lib/libndarray.a # BJ_UCD jeremic@ucdavis.edu
MATMOD_LIBRARY  = $(HOME)/lib/libmatmod.a  # BJ_UCD jeremic@ucdavis.edu
BJMISC_LIBRARY  = $(HOME)/lib/libBJmisc.a  # BJ_UCD jeremic@ucdavis.edu
LAPACK_LIBRARY  = $(HOME)/lib/libLapack.a
ITPACK_LIBRARY  = $(HOME)/lib/libItpack.a
#CLAPACK_LIBRARY  = $(HOME)/OpenSees/OTHER/CLAPACK-3.1.1/lapack_MAC.a
CLAPACK_LIBRARY  = 
#CLBLAS_LIBRARY  = $(HOME)/OpenSees/OTHER/CLAPACK-3.1.1/blas_MAC.a
CLBLAS_LIBRARY  = 
#LIBF2C_LIBRARY  = $(HOME)/OpenSees/OTHER/CLAPACK-3.1.1/blas_MAC.a
LIBF2C_LIBRARY  = 
SUPERLU_LIBRARY = $(HOME)/lib/libSuperLU.a
ARPACK_LIBRARY  = $(HOME)/lib/libArpack.a
AMD_LIBRARY  = $(HOME)/lib/libAMD.a
UMFPACK_LIBRARY = $(HOME)/lib/libUmfpack.a
METIS_LIBRARY   = $(HOME)/lib/libMetis.a
TCL_LIBRARY = -framework Tcl -framework Tk
CSPARSE_LIBRARY = $(HOME)/lib/libCSparse.a

OPTIMIZATION_LIBRARY = 
RELIABILITY_LIBRARY = 

#OPTIMIZATION_LIBRARY = $(HOME)/lib/libOptimization.a
#RELIABILITY_LIBRARY = $(HOME)/lib/libReliability.a

# WATCH OUT .. These libraries are removed when 'make wipe' is invoked.
WIPE_LIBS	= $(FE_LIBRARY) \
		$(NDARRAY_LIBRARY) \
		$(MATMOD_LIBRARY) \
		$(LAPACK_LIBRARY) \
		$(AMD_LIBRARY) \
		$(BLAS_LIBRARY) \
		$(CLBLAS_LIBRARY) \
		$(SUPERLU_LIBRARY) \
		$(ARPACK_LIBRARY) \
		$(UMFPACK_LIBRARY) \
	        $(METIS_LIBRARY) \
	        $(CSPARSE_LIBRARY) \
	        $(ITPACK_LIBRARY)

# %---------------------------------------------------------%
# | SECTION 4: COMPILERS                                    |
# |                                                         |
# | The following macros specify compilers, linker/loaders, |
# | the archiver, and their options.  You need to make sure |
# | these are correct for your system.                      |
# %---------------------------------------------------------%

# Compilers
ifeq ($(PROGRAMMING_MODE), SEQUENTIAL)
CC++	= g++
CC      = gcc
FC	= /usr/local/bin/gfortran
LINKER          = $(CC++)
LINKFLAGS       =  -Wl 
#LINKFLAGS       =  -Wl
else
CC++	= /usr/local/openmpi-1.2.6/bin/mpic++
CC      = /usr/local/openmpi-1.2.6/bin/mpicc
FC	= /usr/local/openmpi-1.2.6/bin/mpif77
LINKER          = $(CC++)
LINKFLAGS       =  -L/usr/local/openmpi-1.2.6/lib /usr/local/openmpi-1.2.6/lib/libmpi.a
#LINKFLAGS       =  -Wl,-u,_munmap -Wl,-multiply_defined,suppress -Wl,-u,_mmap -Wl,-multiply_defined,suppress 
endif

AR		= ar 
ARFLAGS		= cqls
RANLIB		= ranlib
RANLIBFLAGS     =

# Compiler Flags
#
# NOTES:
#    C++ FLAGS TAKE need _UNIX or _WIN32 for preprocessor dircetives
#         - the _WIN32 for the Windows95/98 or NT operating system.
#    C FLAGS used -DUSE_VENDOR_BLAS (needed in SuperLU) if UNIX in C++ FLAGS
#

OS_FLAG = -D_MACOSX

# modified as optimizaton currently causing problems with Steeln01 code
ifeq ($(DEBUG_MODE), DEBUG)

C++FLAGS         = -Wall -D_LINUX -D_UNIX  -D_TCL85  -D_AMDn $(OS_FLAG) -D_HTTPS \
	$(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(DEBUG_FLAG) $(MUMPS_FLAG) \
	$(PROGRAMMING_FLAG) -g -O0 -ffloat-store 
CFLAGS          = -Wall -O0 -g
FFLAGS          = -Wall -O -DCUBLAS -x f77-cpp-input --no-second-underscore


# Linker
LINKER          = $(CC++)
LINKFLAGS       = -g -pg

else

C++FLAGS         = -Wall -D_LINUX -D_UNIX  -D_TCL85  -D_AMDn -D_MACOSX -D_HTTPS \
	$(GRAPHIC_FLAG) $(RELIABILITY_FLAG) $(DEBUG_FLAG) $(MUMPS_FLAG) \
	$(PROGRAMMING_FLAG) -O3 -ffloat-store -D_NO_PARALLEL_FILESYSTEM
CFLAGS          = -Wall -O2
FFLAGS          = -Wall -O -DCUBLAS -x f77-cpp-input --no-second-underscore

# Linker

endif

# Misc
MAKE		= make
CD              = cd
ECHO            = echo
RM              = rm
RMFLAGS         = -f
SHELL           = /bin/sh

# %---------------------------------------------------------%
# | SECTION 5: COMPILATION                                  |
# |                                                         |
# | The following macros specify the macros used in         |
# | to compile the source code into object code.            |
# %---------------------------------------------------------%

.SUFFIXES:
.SUFFIXES:	.C .c .f .f90 .cpp .o .cpp

#
# %------------------%
# | Default command. |
# %------------------%
#
.DEFAULT:
	@$(ECHO) "Unknown target $@, try:  make help"
#
# %-------------------------------------------%
# |  Command to build .o files from .f files. |
# %-------------------------------------------%
#

.cpp.o:
	@$(ECHO) Making $@ from $<
	$(CC++) $(C++FLAGS) $(INCLUDES) -c $< -o $@

.C.o:
	@$(ECHO) Making $@ from $<
	$(CC++) $(C++FLAGS) $(INCLUDES) -c $< -o $@
.c.o:
	@$(ECHO) Making $@ from $<
	$(CC) $(CFLAGS) -c $< -o $@
.f.o:      
	@$(ECHO) Making $@ from $<
	$(FC) $(FFLAGS) -c $< -o $@

# %---------------------------------------------------------%
# | SECTION 6: OTHER LIBRARIES                              |
# |                                                         |
# | The following macros specify other libraries that must  |
# | be linked with when creating executables. These are     |
# | platform specific and typically order does matter!!     |
# %---------------------------------------------------------%

HAVE_SCALAPACK = YES
SCALAPACK_INCLUDE =
SCALAPACK_LIB =

ifeq ($(PROGRAMMING_MODE), SEQUENTIAL)
HAVE_SCALAPACK = NO
endif

ifeq ($(HAVE_SCALAPACK), YES)
SCALAP_DIR = /usr/local/scalapack_installer_0.91
SCALAP = YES
SCALAP_FLAG = -D_SCALAP

SCALAP_LIB_DIR = $(SCALAP_DIR)/lib

SCALAP_LIB  = -L$(SCALAP_LIB_DIR) \
	-lscalapack -lreflapack -lrefblas \
	$(SCALAP_LIB_DIR)/blacs.a \
	$(SCALAP_LIB_DIR)/blacsf77.a \
	$(SCALAP_LIB_DIR)/blacs.a  
endif

HAVE_MUMPS = YES
MUMPS_INCLUDE =
MUMPS_LIB = 

ifeq ($(PROGRAMMING_MODE), SEQUENTIAL)
HAVE_MUMPS = NO
endif

ifeq ($(HAVE_MUMPS), YES)

MUMPS_DIR = /usr/local/MUMPS_4.7.3
MUMPS = YES
MUMPS_FLAG = -D_MUMPS -D_OPENMPI
MUMPS_LIB = -L$(MUMPS_DIR)/lib \
	$(FE)/system_of_eqn/linearSOE/mumps/MumpsSOE.o \
	$(FE)/system_of_eqn/linearSOE/mumps/MumpsSolver.o \
	$(FE)/system_of_eqn/linearSOE/mumps/MumpsParallelSOE.o \
	$(FE)/system_of_eqn/linearSOE/mumps/MumpsParallelSolver.o \
	-ldmumps -lpord \


MUMPS_INCLUDE = -I$(MUMPS_DIR)/include

endif


MACHINE_LINKLIBS  = -L$(BASE)/lib \
		-L$(HOME)/lib 

HAVE_CUDA = NO
CUDA_DIR = 
CUDA_FLAG =
CUDA_LIB =

ifeq ($(HAVE_CUDA),YES)
CUDA_DIR = /usr/local/cuda
CUDA_FLAG = -D_CUDA
#CUDA_LIB = -L$(CUDA_DIR)/lib  -lcublas
CUDA_LIB = $(FE)/system_of_eqn/linearSOE/bandGEN/BandGenLinSOE_Single.o \
	$(FE)/system_of_eqn/linearSOE/bandGEN/BandGenLinLapackSolver_Single.o \
	$(FE)/system_of_eqn/linearSOE/bandGEN/sgbsv.o \
	$(FE)/system_of_eqn/linearSOE/bandGEN/sgbtrs.o \
	$(FE)/system_of_eqn/linearSOE/bandGEN/sgbtrf.o \
	$(FE)/system_of_eqn/linearSOE/bandGEN/sgbtf2.o \
	$(FE)/system_of_eqn/linearSOE/bandGEN/slaswp.o \
	-L$(CUDA_DIR)/lib -cublas -lcuda \
	-L/usr/local/scalapack_installer_0.91/lib \
	-lrefblas

CUDA_INCLUDE = -I$(CUDA_DIR)/include
endif


#FORTO = $(LAPACKdir)/fortran.o
FORTO = 

MACHINE_NUMERICAL_LIBS  =  \
		$(ARPACK_LIBRARY) \
		$(SUPERLU_LIBRARY) \
		$(UMFPACK_LIBRARY) \
	        $(AMD_LIBRARY) \
	        $(ITPACK_LIBRARY) \
	        $(LAPACK_LIBRARY) \
	        $(CLAPACK_LIBRARY) \
	        $(CLBLAS_LIBRARY) \
		$(CSPARSE_LIBRARY) \
		$(BLAS_LIBRARY) \
		$(CBLAS_LIBRARY) \
		$(GRAPHIC_LIBRARY)\
		$(RELIABILITY_LIBRARY)\
		$(OPTIMIZATION_LIBRARY)\
		$(FORTO) \
		$(CUDA_LIB) \
		-ldl -L/usr/local/lib /usr/local/lib/libgfortran.a


MACHINE_SPECIFIC_LIBS = $(AGL_OBJS) -lssl

PARALLEL_LIB = 	$(FE)/system_of_eqn/linearSOE/sparseGEN/DistributedSuperLU.o \
	     $(DISTRIBUTED_SUPERLU_LIBRARY) \
	$(FE)/system_of_eqn/linearSOE/sparseGEN/DistributedSparseGenColLinSOE.o \
	$(FE)/system_of_eqn/linearSOE/sparseGEN/SparseGenColLinSOE.o \
	$(LAPACK_LIB) $(DISTRIBUTED_SUPERLU_LIBRARY) $(MUMPS_LIB) $(SCALAP_LIB) $(METIS_LIBRARY) \
	-L/usr/local/openmpi-1.2.6/lib -lmpi_cxx -lmpi_f77 -lmpi -lopen-rte -lopen-pal

ifeq ($(PROGRAMMING_MODE), PARALLEL_INTERPRETERS)

PARALLEL_LIB = 	$(FE)/system_of_eqn/linearSOE/sparseGEN/DistributedSuperLU.o \
	$(FE)/system_of_eqn/linearSOE/sparseGEN/DistributedSparseGenColLinSOE.o \
	$(DISTRIBUTED_SUPERLU_LIBRARY) $(MUMPS_LIB) $(SCALAP_LIB) $(METIS_LIBRARY) \
	-L/usr/local/openmpi-1.2.6/lib -lmpi_cxx -lmpi_f77 -lmpi -lopen-rte -lopen-pal

endif


# %---------------------------------------------------------%
# | SECTION 7: INCLUDE FILES                                |
# |                                                         |
# | The following macros specify include files needed for   |
# | compilation.                                            |
# %---------------------------------------------------------%

ifeq ($(PROGRAMMING_MODE),  SEQUENTIAL)
MACHINE_INCLUDES        = -I/usr/include \
			  -I/usr/local/include \
			  -I$(BASE)/include \
			  -I/usr/include/cxx \
			  -I$(HOME)/include \
			  -I$(HOME)/blitz \
			  -I$(CUDA_DIR)/include
else
MACHINE_INCLUDES        = -I/usr/include \
			  -I/usr/local/include \
			  -I$(BASE)/include \
			  -I/usr/include/cxx \
			  -I$(SUPERLU_DISTdir) \
			  -I$(HOME)/include \
			  -I$(HOME)/blitz \
			  $(MUMPS_INCLUDE) \
			  -I$(CUDA_DIR)/include
endif

# this file contains all the OpenSees/SRC includes
include $(FE)/Makefile.incl

TCL_INCLUDES = -I/Library/Frameworks/Tcl.framework/Headers -I/Library/Frameworks/Tk.framework/Headers

INCLUDES = $(TCL_INCLUDES) $(FE_INCLUDES) $(MACHINE_INCLUDES)








